#include "DictProducer.h"
#include "DirScanner.h"
#include <iostream>
#include <fstream>
#include <sstream>

DictProducer::DictProducer(Configuration *config)
: _config(config), _splitTool(nullptr), _files(), _dict(), _index(), _forZh(false){
    getFiles(); //获取文件集合
}

DictProducer::DictProducer(Configuration *config, SplitTool *splitTool)
: _config(config), _splitTool(splitTool), _files(), _dict(), _index(), _forZh(true){
    getFiles(); //获取文件集合
}

DictProducer::~DictProducer(){}

static void cleanLine(string &line){
    for(auto &ch: line){
        if(!isalpha(ch)){
            ch = ' ';
        }
        else if(isupper(ch)){
            ch = tolower(ch);
        }
    }
}

void DictProducer::buildEnDict(){
    if(_forZh){
        return;
    }
    set<string> _stopWords = _config->getStopWordList();
    for(auto &filename: _files){
        ifstream ifs(filename);
        if(!ifs){
            cerr << "ifs open file: " << filename << " failed!\n";
            return;
        }
        string line, word;
        while(getline(ifs, line)){
            cleanLine(line);
            istringstream iss(line);
            while(iss >> word){
                if(_stopWords.count(word) == 0){
                    _dict[word]++;
                }
            }
        }
        ifs.close();
    }
}

//创建中文词典
void DictProducer::buildCnDict(){
    if(!_forZh){
        return;
    }
    //去停用词
    set<string> _stopWords = _config->getStopWordList();
    for(auto &filename: _files){
        ifstream ifs(filename);
        if(!ifs){
            cerr << "ifs open file: " << filename << " failed!\n";
            return;
        }
        //把一篇语料全部读到一个字符串里
        string line, sentence;
        vector<string> words;
        while(getline(ifs, line)){
            sentence += line;
        }
        //使用jieba进行分词
        words = _splitTool->cut(sentence);
        //统计词频
        for(auto &word: words){
            if(_stopWords.count(word) == 0 && word != " "){
                _dict[word]++;
            }
        }
        ifs.close();
    }
}

//创建英文索引表
void DictProducer::createEnIndex(){
    int pos = 0;
    for(auto &i: _dict){
        for(auto j: i.first){
            string s;
            s.push_back(j);
            _index[s].insert(pos);
        }
        pos++;
    }
}

static size_t nBytesCode(const char ch){
    if(ch & (1 << 7)){
        int nBytes = 1;
        for(int idx = 0; idx != 6; ++idx){
            if(ch & (1 << (6 - idx))){
                ++nBytes;
            }
            else{
                break;
            }
        }
        return nBytes;
    }
    return 1;
}

//创建中文索引表
void DictProducer::createCnIndex(){
   int pos = 0; //使用变量pos代替vector的下标，减少一次转换
   for(auto &i: _dict){
       for(size_t j = 0; j < i.first.size();){
           size_t size = nBytesCode(i.first[j]); 
           string s = i.first.substr(j, size);
           j += size;
           _index[s].insert(pos);
       }
       pos++;
   }
}

void DictProducer::storeDict(const string &filepath){
    ofstream ofs(filepath);
    if(ofs.is_open()){
        for(const auto &pair: _dict){
            ofs << pair.first << " " << pair.second << "\n";
        }
        ofs.close();
    }
    else{
        cerr << "无法打开文件进行写入: " << filepath << "\n";
    }
}

void DictProducer::storeIndex(const string &filepath){
    ofstream ofs(filepath);
    if(ofs.is_open()){
        for(const auto &pair: _index){
            ofs << pair.first << " ";
            for(const auto &i: pair.second){
                ofs << i << " ";
            }
            ofs << "\n";
        }
        ofs.close();
    }
    else{
        cerr << "无法打开文件进行写入: " << filepath << "\n";
    }
}

void DictProducer::showFiles() const{
    for(auto &i: _files){
        cout << i << " ";
    }
    cout << "\n";
}

void DictProducer::showDict() const{
    for(auto &i: _dict){
        cout  << i.first << ": " << i.second << "\n";
    }
}

void DictProducer::showIndex() const{
    for(auto &i: _index){
        cout  << i.first << ": (";
        for(auto &j: i.second){
            cout << j << ",";
        }
        cout << ")\n";
    }
}

void DictProducer::getFiles(){
    string dir;
    if(!_forZh){
        dir = _config->parseConfig("CORPUS_EN");
    }
    else{
        dir = _config->parseConfig("CORPUS_ZH");
    }
    DirScanner scanner;
    scanner(dir);
    _files = scanner.files();
}
